Transform Weather Sensor Data from Waggle to an Xarray Dataset + Plot with ACT

Imports

import sage_data_client
from bokeh.models.formatters import DatetimeTickFormatter
import hvplot.pandas
import hvplot.xarray
import holoviews as hv
import cartopy.crs as ccrs
import cartopy.feature as cfeature
import xarray as xr
import matplotlib.pyplot as plt
from metpy.plots import USCOUNTIES
import act
import numpy as np
import pandas as pd
import warnings
from bokeh.models import DatetimeTickFormatter

def apply_formatter(plot, element):
    plot.handles['xaxis'].formatter = DatetimeTickFormatter(hours='%m/%d/%Y \n %H:%M',
                                                            minutes='%m/%d/%Y \n %H:%M',
                                                            hourmin='%m/%d/%Y \n %H:%M',
                                                            days='%m/%d/%Y \n %H:%M',
                                                            months='%m/%d/%Y \n %H:%M')
xr.set_options(keep_attrs=True)
warnings.filterwarnings("ignore")
hv.extension("bokeh")

Query for the available Data

wxt_df = sage_data_client.query(
    start="-3h",
    filter={
        "sensor": "vaisala-wxt536"
    }
)

Configure Helper Functions and Renaming Conventions

The renaming is required due to . notations being problematic when working with both Pandas and Xarray data structures.

variable_rename_dict = {'wxt.env.humidity':'air_humidity',
                        'wxt.env.pressure':'air_pressure',
                        'wxt.env.temp':'air_temperature',
                        'wxt.heater.temp':'heater_temperature',
                        'wxt.heater.volt':'heater_voltage',
                        'wxt.rain.accumulation':'rain_accumulation',
                        'wxt.wind.direction':'wind_direction',
                        'wxt.wind.speed':'wind_speed',
                        'sys.gps.lat':'latitude',
                        'sys.gps.lon':'longitude',
                    }

def generate_data_array(df, variable, rename_variable_dict=variable_rename_dict):
    new_variable_name = rename_variable_dict[variable]
    df_variable= df.loc[df.name == variable]
    ds = df_variable.to_xarray().rename({'value':new_variable_name,
                                         'timestamp':'time',
                                         'meta.vsn':'node'})
    ds[new_variable_name].attrs['units'] = df_variable['meta.units'].values[0]
    ds['time'] = pd.to_datetime(ds.time)
    ds.attrs['datastream'] = ds.node.values[0]
    return ds[[new_variable_name]]

def generate_dataset(df, variables, rename_variable_dict=variable_rename_dict):
    reindexed = df.set_index(['meta.vsn', 'timestamp'])
    return xr.merge([generate_data_array(reindexed, variable) for variable in variables])

Transform the Data to Xarray

wxt_variables = wxt_df.name.unique()
wxt_variables

array(['wxt.env.humidity', 'wxt.env.pressure', 'wxt.env.temp',
       'wxt.heater.temp', 'wxt.heater.volt', 'wxt.rain.accumulation',
       'wxt.wind.direction', 'wxt.wind.speed'], dtype=object)

wxt_ds = generate_dataset(wxt_df, wxt_variables).squeeze()
wxt_ds

<xarray.Dataset> Size: 85MB
Dimensions:             (node: 4, time: 321547)
Coordinates:
  * node                (node) object 32B 'W01B' 'W067' 'W08D' 'W099'
  * time                (time) datetime64[ns] 3MB 2024-04-10T13:05:11.5758620...
Data variables:
    air_humidity        (node, time) float64 10MB nan 91.4 nan ... nan 62.4 nan
    air_pressure        (node, time) float64 10MB nan 997.4 nan ... 990.9 nan
    air_temperature     (node, time) float64 10MB nan 8.6 nan ... nan 12.1 nan
    heater_temperature  (node, time) float64 10MB nan 12.9 nan ... nan 20.3 nan
    heater_voltage      (node, time) float64 10MB nan 11.4 nan ... nan 23.9 nan
    rain_accumulation   (node, time) float64 10MB nan 342.1 nan ... 311.0 nan
    wind_direction      (node, time) float64 10MB nan 314.0 nan ... 130.0 nan
    wind_speed          (node, time) float64 10MB nan 1.2 nan ... nan 1.8 nan
Attributes:
    datastream:  W01B

xarray.Dataset

• Dimensions:
  • node: 4
  • time: 321547
• Coordinates: (2)
  • node
    (node)
    object
    'W01B' 'W067' 'W08D' 'W099'

    array(['W01B', 'W067', 'W08D', 'W099'], dtype=object)

  • time
    (time)
    datetime64[ns]
    2024-04-10T13:05:11.575862060 .....

    array(['2024-04-10T13:05:11.575862060', '2024-04-10T13:05:11.589892116',
           '2024-04-10T13:05:11.657885287', ..., '2024-04-10T16:05:11.732838058',
           '2024-04-10T16:05:11.749451078', '2024-04-10T16:05:11.808185780'],
          dtype='datetime64[ns]')

• Data variables: (8)
  • air_humidity
    (node, time)
    float64
    nan 91.4 nan 91.4 ... nan 62.4 nan

    units :

    percent

    array([[ nan, 91.4,  nan, ...,  nan,  nan,  nan],
           [ nan,  nan,  nan, ...,  nan,  nan,  nan],
           [ nan,  nan,  nan, ...,  nan,  nan,  nan],
           [60.5,  nan, 60.5, ...,  nan, 62.4,  nan]])

  • air_pressure
    (node, time)
    float64
    nan 997.4 nan ... nan 990.9 nan

    units :

    hectoPascal

    array([[  nan, 997.4,   nan, ..., 996.9,   nan, 996.9],
           [  nan,   nan,   nan, ...,   nan,   nan,   nan],
           [  nan,   nan,   nan, ...,   nan,   nan,   nan],
           [991.2,   nan, 991.2, ...,   nan, 990.9,   nan]])

  • air_temperature
    (node, time)
    float64
    nan 8.6 nan 8.6 ... nan 12.1 nan

    units :

    degree Celsius

    array([[ nan,  8.6,  nan, ..., 16.3,  nan, 16.3],
           [ nan,  nan,  nan, ...,  nan,  nan,  nan],
           [ nan,  nan,  nan, ...,  nan,  nan,  nan],
           [11.2,  nan, 11.2, ...,  nan, 12.1,  nan]])

  • heater_temperature
    (node, time)
    float64
    nan 12.9 nan 12.9 ... nan 20.3 nan

    units :

    degree Celsius

    array([[ nan, 12.9,  nan, ..., 20.1,  nan, 20.1],
           [ nan,  nan,  nan, ...,  nan,  nan,  nan],
           [ nan,  nan,  nan, ...,  nan,  nan,  nan],
           [15.1,  nan, 15.1, ...,  nan, 20.3,  nan]])

  • heater_voltage
    (node, time)
    float64
    nan 11.4 nan 11.4 ... nan 23.9 nan

    units :

    volts

    array([[ nan, 11.4,  nan, ..., 12. ,  nan, 12. ],
           [ nan,  nan,  nan, ...,  nan,  nan,  nan],
           [ nan,  nan,  nan, ...,  nan,  nan,  nan],
           [23.8,  nan, 23.8, ...,  nan, 23.9,  nan]])

  • rain_accumulation
    (node, time)
    float64
    nan 342.1 nan ... nan 311.0 nan

    units :

    milimeters

    array([[   nan, 342.13,    nan, ..., 342.13,    nan, 342.13],
           [   nan,    nan,    nan, ...,    nan,    nan,    nan],
           [   nan,    nan,    nan, ...,    nan,    nan,    nan],
           [310.98,    nan, 310.98, ...,    nan, 310.98,    nan]])

  • wind_direction
    (node, time)
    float64
    nan 314.0 nan ... nan 130.0 nan

    units :

    degrees

    array([[ nan, 314.,  nan, ..., 339.,  nan, 339.],
           [ nan,  nan,  nan, ...,  nan,  nan,  nan],
           [ nan,  nan,  nan, ...,  nan,  nan,  nan],
           [301.,  nan, 301., ...,  nan, 130.,  nan]])

  • wind_speed
    (node, time)
    float64
    nan 1.2 nan 1.2 ... 1.8 nan 1.8 nan

    units :

    meters per second

    array([[nan, 1.2, nan, ..., 2.7, nan, 2.7],
           [nan, nan, nan, ..., nan, nan, nan],
           [nan, nan, nan, ..., nan, nan, nan],
           [0.1, nan, 0.1, ..., nan, 1.8, nan]])

• Indexes: (2)
  • node
    PandasIndex

    PandasIndex(Index(['W01B', 'W067', 'W08D', 'W099'], dtype='object', name='node'))

  • time
    PandasIndex

    PandasIndex(DatetimeIndex(['2024-04-10 13:05:11.575862060',
                   '2024-04-10 13:05:11.589892116',
                   '2024-04-10 13:05:11.657885287',
                   '2024-04-10 13:05:11.684567661',
                   '2024-04-10 13:05:11.751494053',
                   '2024-04-10 13:05:11.769391196',
                   '2024-04-10 13:05:11.835805894',
                   '2024-04-10 13:05:11.877475887',
                   '2024-04-10 13:05:11.911256428',
                   '2024-04-10 13:05:11.992010659',
                   ...
                   '2024-04-10 16:05:11.430997731',
                   '2024-04-10 16:05:11.505818409',
                   '2024-04-10 16:05:11.509688425',
                   '2024-04-10 16:05:11.581205841',
                   '2024-04-10 16:05:11.589965268',
                   '2024-04-10 16:05:11.656330954',
                   '2024-04-10 16:05:11.669461852',
                   '2024-04-10 16:05:11.732838058',
                   '2024-04-10 16:05:11.749451078',
                   '2024-04-10 16:05:11.808185780'],
                  dtype='datetime64[ns]', name='time', length=321547, freq=None))

• Attributes: (1)

  datastream :

  W01B

Resample the data to minute requency

minute_ds = wxt_ds.resample(time='1T').mean()

Visualize using hvplot

meteogram_variables = ['air_temperature', 'air_humidity', 'wind_speed', 'wind_direction']
plots = []
for variable in meteogram_variables:
    plots.append(wxt_ds[variable].hvplot.line(label='10 Hz Data') * 
                 minute_ds[variable].hvplot.line(label='1 Minute Data'))
hv.Layout(plots).cols(2)

Plot Using the Atmospheric Data Community Toolkit (ACT)

WindDisplay = act.plotting.WindRoseDisplay(minute_ds, figsize=(8, 10), subplot_shape=(1,))
WindDisplay.plot(
    'wind_direction', 'wind_speed', spd_bins=np.linspace(0, 25, 5), num_dirs=30, tick_interval=2, subplot_index=(0,)
)

<PolarAxes: title={'center': 'W01B on 20240410'}>